#!/usr/bin/env python 
# -*- coding:utf-8 -*-

import requests
import json
import time
import pymysql

URL_FORMAT = 'http://sclub.jd.com/comment/productPageComments.action'
commodity_id = 100005332318

def main():
    """
    京东评论爬虫
    :return:
    """
    for x in range(8, 100):
        url = URL_FORMAT.format(x)
        params = {
            'productId': commodity_id,
            'score': 0,
            'sortType': 5,
            'page': x,
            'pageSize': 10,
            'isShadowSku': 0,
            'rid': 0,
            'fold': 1
        }
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36',
            'Referer': 'https://item.jd.com/%d.html' % commodity_id
        }
        print('开始爬取第%d页' % x)
        timeout = 3
        while timeout > 0:
            timeout -= 1
            try:
                response = requests.get(url, headers=headers, params=params)
                comments = json.loads(response.text)['comments']
                if not comments:
                    print('未获取到评论')
                    break
                comment_list = []
                for comment in comments:
                    id = comment['id']
                    nickname = comment['nickname']
                    content = comment['content']
                    creationTime = comment['creationTime']
                    isTop = '是' if comment['isTop'] else '否'
                    productColor = comment['productColor']
                    productSize = comment['productSize']
                    userClientShow = comment['userClientShow']
                    productSale = comment['productSales'][0].get('saleValue', '') if comment['productSales'] else ''
                    comment_list.append((id, nickname, content, creationTime, isTop, productColor, productSize, userClientShow, productSale, commodity_id))
                save(comment_list)
                print('第%d页爬取结束' % x)
                time.sleep(5)
                break
            except Exception:
                print('请求错误')
                time.sleep(3)
        if timeout <= 0:
            print('第%d页爬取失败' % x)
def save(comment_list):
    conn = pymysql.connect(host='127.0.0.1', port=3306, user='root', password='root',
                           database='jd_comment')
    cursor = conn.cursor()
    sql = """
       REPLACE INTO comment(id, nickname, content, creation_time, is_top, product_color, product_size, user_client_show, product_sale, commodity_id) 
       VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
    """
    cursor.executemany(sql, comment_list)
    conn.commit()
    cursor.close()
    conn.close()
if __name__ == '__main__':
    main()
